'''Provides a bibliography class with methods to load BibTeX data.

@copyright: (c) 2009-2010, Benjamin Kalish

@license:
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    General Public License for more details.

@todo:
    write module level documentation
@todo:
    test all file and string input with a variety of string encodings.
'''
# standard libraries
import codecs
import copy
import json
import locale
import os
import re
import subprocess
import sys
import tempfile
import types
import urllib
import urlparse
import warnings

# 3rd party libraries
from simpleparse.error import ParserSyntaxError

# BKN libraries
from bkn.bibtex._bibtex_parser import BIBTEX_PARSER
from bkn.bibtex._bibtex_parser import BibtexProcessor
from bkn.bibtex.utilities import assert_objects_are_unicode
from bkn.bibtex.utilities import ensure_objects_are_unicode
from bkn.bibtex.error import AttributeCollisionWarning
from bkn.bibtex.error import BadAttributeNameError
from bkn.bibtex.error import BibFileSyntaxError
from bkn.bibtex.error import BibTeXError
from bkn.bibtex.error import BKNBibtexError
from bkn.bibtex.error import BKNBibtexWarning
from bkn.bibtex.error import FormatNotSupportedError
from bkn.bibtex.error import InvalidRecordError
from bkn.bibtex.error import InvalidReferenceError
from bkn.bibtex.error import InvalidResponseError
from bkn.bibtex.error import RecordNotFoundError
from bkn.bibtex.utilities import which


# Module level constants

BIBTEX_FIELDS = ['address', 'annote', 'author', 'booktitle', 'chapter', 
    'crossref', 'edition', 'editor', 'howpublished', 'institution', 'journal', 
    'key', 'month', 'note', 'number', 'organization', 'pages', 'publisher',
    'school', 'series', 'title', 'type', 'volume', 'year']
'''A list of the standard fields used by BibTeX.

Field names are taken from I{BibTeXing} by Oren Patashnick (1988).

@type: C{list} of C{str}s
@note: this constant was formerly named STANDARD_FIELDS.
'''

BIBTEX_TYPES = ['article', 'book', 'booklet', 'conference', 'inbook',
    'incollection', 'inproceedings', 'manual', 'mastersthesis', 'misc',
    'phdthesis', 'proceedings', 'techreport', 'unpublished']
'''A list of the standard entry types used by BibTeX

Entry types are taken from I{BibTeXing} by Oren Patashnick (1988).

@type: C{list} of C{str}s
'''

BIBTEX_WARNING_REGEX = re.compile(r'Warning--(.*) in (.*)$')
'''A regular expression which matches the warning lines which BibTeX prints to standard out.

The expression contains two groups: the first group describes the
problem, the second group contains citation key of the entry in which it
was encountered.

@type: C{re.RegexObject}'''

DEFAULT_ENCODING = locale.getpreferredencoding()
'''The default text encoding as returned by locale.getpreferredencoding().'''

bib_cache = {}
'''A cache of bibs which have been opened.

This caches will be used when looking up an record by id using
bibliography_from_id() or entry_from_id().

@type: C{dict}
@deprecated:
    This caching method will be dropped in future revisions.
'''


# Module level functions

def bibliography_from_id(dataset_id):
    '''Returns a bibliography based on its id.
    @rtype: L{Bibliography}
    @todo: consider revising or replacing the caching system used here.
    @todo: add error handling
    '''
    ensure_objects_are_unicode(dataset_id)
    try:
        bib = bib_cache[dataset_id]
    except KeyError:
        bib = bib_cache[dataset_id] = Bibliography(file=dataset_id)
    return bib
     
def record_from_id(global_id):
    '''Returns a bibliography entry given the global id of the record.
    
    Unlike the L{resolve_reference()<Bibliography.resolve_reference>} method, this is a module level
    function and does not require a C{Bibliography} instance. As a
    consequence it can only resolve global ids.
    
    @param global_id:
        the global id of a record. This should be just the records id
        and should not include the "@@" that precedes a global id in
        BibJSON references.
    @rtype: L{Record}
    @todo: document return values and exceptions
    @todo: use appropriate error classes
    @warning:
         Do not confuse this module level method with the
         L{record_from_id()<Bibliography.record_from_id>} method in the L{Bibliography} class.
    '''
    ensure_objects_are_unicode(global_id)
    parse_result = urlparse.urlparse(global_id)
    if parse_result.scheme == '':
        raise InvalidReferenceError(
            '"' + global_id + '" is not a valid reference. ' +
            'Valid global references must be URIs.')
    uri_without_fragment, fragment = urlparse.urldefrag(global_id)
    try:
        response = urllib.urlopen(uri_without_fragment)
    except IOError as ioerror:
        # reference was valid but could not be resolved
        raise RecordNotFoundError(
            '"' + global_id + '" appears to be a valid reference but could ' +
            'not be resolved. This could be a problem with your network or ' +
            'the service meant to be hosting this resource. (IOError:' +
            str(ioerror) + ')')
    try:
        py_object = json.load(response)
    except:
        raise Exception('The document retrieved from ' + uri_without_fragment +
            ' could not be parsed as JSON.')
    if py_object.has_key(u'id'):
        # an individual record was returned
        result = Record(attribute_map=py_object)
        # we should set parent_bib_id here if we can
        return result
    elif py_object.has_key('recordList'):
        # get dataset id
        dataset_id = py_object[u'dataset'][u'id']
        for record in py_object[u'recordList']:
            if dataset_id + record[u'id'] == global_id:
                result = Record(attribute_map=record)
                result.parent_bib_id = dataset_id
                return result
        raise RecordNotFoundError(
            'No record with id "' + global_id + '" was found in the ' +
            'record list retrieved from ' + uri_without_fragment + '.')
    else:
        raise InvalidResponseError(
            'Invalid response. The retrieved resource does not appear to ' +
            'be valid BibJSON.')

def entry_from_id(dataset_id, record_id):
    '''
    @deprecated:
        use L{record_from_id()} instead.
    @rtype: L{Record}
    '''
    ensure_objects_are_unicode(dataset_id, record_id)
    warnings.warn(
        "The entry_from_id() method has been deprecated. Use record_from_id() instead.", 
        DeprecationWarning, stacklevel=2)
    return record_from_id(dataset_id + record_id)

def _sort_with_standard_fields_first(attribute_names):
    '''Sorts a list of attribute names so that standard BibTeX fields
    are listed first and nonstandard fields are listed second. Within
    each category fields will be listed alphabetically.
    '''
    ensure_objects_are_unicode(*attribute_names)
    nonstandard_attributes = list(set(attribute_names) - set(BIBTEX_FIELDS))
    standard_attributes = list(set(attribute_names) - set(nonstandard_attributes))
    nonstandard_attributes.sort()
    standard_attributes.sort()
    return standard_attributes + nonstandard_attributes


# Class definitions

class Record:
    '''A single record in the dataset, usually a bibliography entry.
    
    The preferred method to access to a Record's attributes is to use bracket
    notation, just as the built in dictionary type does, so you can do:
    
        >>> e = Record(u'key', bibtype=u'article')
        >>> e[u'title'] = 'Of Mice and Men'
        >>> e[u'title']
        'Of Mice and Men'
    
    An empty string will be returned if you try to access an undefined
    attribute. Attribute names accessed and set this way are case insensitive.
    
    @warning:
       While the Record class does handle attribute access using the
       bracket notation it is not a mapping object. If you need to use
       mapping object methods you can access a Records's attribute
       dictionary directly:
           
           >>> field_names = record.attribute_map.keys()
       
       It is, however, important that any new keys added to the
       dictionary in this way be lowercase, as the following example
       illustrates:
       
           >>> record[u'TITLE'] = 'Spam'
           >>> record.fields[u'TITLE'] = 'Eggs'
           >>> record[u'TITLE']
           'Spam'      # not eggs!
    '''
    def __init__(self, citekey=None, bibtype=None, attribute_map=None):
        '''Initializes a new Record instance.
        
        @param citekey: the local id for this record.
        @type citekey: str
        @param bibtype: the type attribute for this record.
        @type bibtype: str
        @param attribute_map:
             attribute name-value pairs with which to initialize this
             Record.
        @type attribute_map: dict
        @raise BKNBibtexError: if an id for the record has not been specified. 
        
        @todo:
            deprecate 'citekey' and 'bibtype' keyword arguments in
            favor of 'id' and 'type'
        @todo: consider subclassing BKNBibtexError
        '''
        if attribute_map is not None:
            ensure_objects_are_unicode(citekey, bibtype, **attribute_map)
        else:
            ensure_objects_are_unicode(citekey, bibtype)
        
        
        self._parent_bib = None
        '''The bibliography to which this Record belongs.
        @type: L{Bibliography} or C{None}
        '''
        
        self.parent_bib_id = None
        '''The id of the parent bibliography.
        
        This will always be set when the parent bib is set, but it may
        sometimes be useful to set this without setting the parent_bib,
        for example, when loading the parent bib into memory might be
        undesirable.
        '''
        
        self.attribute_map = {}
        '''A dictionary mapping attribute names to their values.
        
        @type: C{dict}
        @warning:
            This attribute should not be modified! New attributes may be
            added or modified using bracket notation:
               >>> record[attribute_name] = value
        '''
        if attribute_map is not None:
            for attribute, value in attribute_map.items():
                self[attribute] = value
        
        if self.attribute_is_blank(u'type'):
            if bibtype is None:
                bibtype = u'UNKNOWN'
            self[u'type'] = bibtype
        
        if self.attribute_is_blank(u'id'):
            if citekey is None:
                raise BKNBibtexError(
                    'You must provide an id, either in the attributes or using the citekey keyword argument.')
                    # consider subclassing BKNBibtexError
            self[u'id'] = citekey
        
    def __str__(self):
        '''Returns an informal string representation of this record
        
        @return:
            The current implementation calls to_bibtex(). (This may
            change in future versions.
        @rtype: C{str}
        '''
        return self.to_bibtex()
        
    #{ Container methods
    
    def __getitem__(self, attribute_name):
        '''Returns the value of an attribute. Called when using bracket notation.
        
            >>> record[u'author']
            ... "Monty Python"
            >>> record[u'title']
            ... "Monty Python and the Holy Grail"
        
        @param attribute_name:
            the name of the attribute whose value should be returned.
        @type attribute_name:
            str
        @return: the value of the requested attribute.
        @rtype: C{object}
        @raises BadAttributeNameError:
            if C{attribute_name} is not a valid attribute name
        '''
        ensure_objects_are_unicode(attribute_name)
        Record.validate_attribute_name(attribute_name)
        name = attribute_name.lower()
        if self.attribute_map.has_key(name):
            result = self.attribute_map[name]
            assert_objects_are_unicode(result)
            return result
        return u''

    def __setitem__(self, attribute_name, attribute_value):
        '''Sets the value of an attribute. Called when using bracket notation.
        
            >>> record[u'id'] = '25'
            >>> record[u'type'] = 'article'
        
        @param attribute_name:
            the name of the attribute whose value should be set.
        @type attribute_name:
            str
        @param attribute_value:
            the value to which the attribute should be set.
        @type attribute_value:
            object
        @rtype: C{None}
        @raises BadAttributeNameError:
            if C{attribute_name} is not a valid attribute name
        @todo:
            use schema to check object attribute_value for allowed
            values.
        @todo:
            this method is long and confusing. it should be refactored.
        '''
        ensure_objects_are_unicode(attribute_name, attribute_value)        
        Record.validate_attribute_name(attribute_name)        
        name = attribute_name.lower() # attribute names are case-insensitive
        
        # Check for duplicates ids
        if self._parent_bib is not None:
            if name == 'id':
                if attribute_value in self._parent_bib.id_to_record_index.keys():
                    if self is not self._parent_bib.id_to_record_index[attribute_value]:
                        raise BKNBibtexError(
                            'A record with id ' + repr(attribute_value) +
                            ' already exists!')
                        # should subclass BKNBibtexError
        
        # validate record:
        self.validate()
        
        old_value = self[name]
        self.attribute_map[name] = attribute_value

        # update id_to_record_index
        if name == 'id':
            if self._parent_bib is not None:
                try:
                    del self._parent_bib.id_to_record_index[old_value]
                except KeyError:
                    pass
                self._parent_bib.id_to_record_index[self[name]] = self
        
    def __delitem__(self, attribute_name):
        '''Removes an attribute from the record. Called when using bracket notation.
        
            >>> del record['issn']
        
        @param attribute_name:
            the name of the attribute to be removed.
        @type attribute_name:
            str
        @rtype: C{None}
        @raises BadAttributeNameError:
            if C{attribute_name} is not a valid attribute name
        '''
        ensure_objects_are_unicode(attribute_name)        
        Record.validate_attribute_name(attribute_name)
        name = attribute_name.lower()
        if self.attribute_map.has_key(name):
            del self.attribute_map[name]
            
    #{ Serialization methods
    
    def to_bibtex(self):
       '''Returns this record serialized as a BibTeX entry.
       
           >>> record.to_bibtex()
           ... "@book{23,
           ...    title = {Of Mice and Men},
           ...    author = {John Steinbeck}
           ...}
       
       @return:
           the BibTeX serialization of this record. (Non-string
           attribute values will be converted to JSON.)
       @rtype: C{str}
       @warning:
             Bibliographies with any attribute values which are not strings
             or integers may not be serialized in the BibTeX format.
       '''       
       keys = _sort_with_standard_fields_first(self.attribute_map.keys())
       
       # This second block constructs the BibTeX output.
       out = u'@' + self[u'type'] + u'{' + self[u'id'] + u',\n'
       for key in keys:
           attribute_value = self.attribute_map[key]
           if not isinstance(attribute_value, types.StringTypes):
               warnings.warn(
                  'The value of the ' + key + ' attribute will be converted' +
                  ' to a string in BibTeX output. The resulting ' +
                  'serialization may result in a loss of information',
                  BKNBibtexWarning, stacklevel=2)
               attribute_value = json.dumps(attribute_value)
           out += '  ' + key + ' = {' + attribute_value + '},\n'
       out += '}\n'
       assert_objects_are_unicode(out)        
       return out

    #{ Information methods
    
    def set_parent_bib(self, bib):
        '''Sets the parent bibliography.
        
        This method also sets parent_bib_id and is prefered to setting
        the attribute directly.
        '''
        self.parent_bib_id = bib.metadata[u'id']
        self._parent_bib = bib
    
    def get_parent_bib(self):
        '''Returns the L{Bibliography} to which this record belongs.
        '''
        if self._parent_bib is not None:
            return self._parent_bib
        elif self.parent_bib_id is not None:
            result = bibliography_from_id(self.parent_bib_id)
            self.set_parent_bib(result)
            return result
        else:
            return None

    def attribute_is_blank(self, attribute_name):
        '''Returns True if the supplied attribute is blank.
        
        Returns True if the supplied attribute has a value of None,
        whitespace, an empty list, or an empty dict.
        
        @param attribute_name:
            the name of the attribute to check.
        @type attribute_name:
            str
        @rtype: C{bool}
        '''
        ensure_objects_are_unicode(attribute_name)        
        Record.validate_attribute_name(attribute_name)
        try:
            test = self[attribute_name].strip()
            if test == '':
                return True
        except AttributeError:
            pass
        if self[attribute_name] in [None, [], {}]:
            return True
        return False

    def attribute_names(self):
       '''Returns a list of the field names for each non-empty field in
       the Record.
       
       @return: the names of all the attributes for this Record.
       @rtype: C{list}
       '''
       result = [k for k in self.attribute_map.keys() 
           if not self.attribute_is_blank(k)]
       assert_objects_are_unicode(*result)
       return result
    
    def fullname(self):
        '''Returns the name of the parent_bib and this records id,
        separated by a colon.
        
        @deprecated:
             Use the global_id() method instead.
        @rtype: C{str}
        '''
        try:
            result = self.parent_bib.id + ':' + self[u'id']
            assert_objects_are_unicode(result)
            return result
        except AttributeError:
            return u'<no parent bib>: ' + self[u'id']

    def global_id(self):
        '''Returns the global id of this record.
        
        @return:
            the global is of this record, i.e. the concatenation of the
            dataset id and the local id for this record.
        @rtype: C{str}
        @todo:
            raise appropriate error when self.parent_bib_id is None
        '''
        result = self.get_parent_bib().metadata[u'id'] + self[u'id']
        assert_objects_are_unicode(result)
        return result

    def names_for_sort(self):
        '''Returns the value of the author attribute if it exists, otherwise returns the value of the editor attribute.
        
        This method may be used when sorting bibliography entries.
        @return:
            the value of the author attribute if it exists, otherwise
            returns the value of the editor attribute.
        @rtype: C{object}
        @note:
            The returned object may be of any type but will most often be
            a string or a list of strings.
        '''
        if not self.attribute_is_blank(u'author'):
            return self[u'author']
        return self[u'editor']

    def values_equal(self, other):
        '''Returns true if this Record has the same content as the given Record.
        
        Distinct Records will always compare as not equal. This method may
        be used to compare Records based upon their content.
        
            >>> r1 = Record(attribute_map={u'id' : '1', u'title' : u'Spam' })
            >>> r2 = Record(attribute_map={u'id' : '1', u'title' : u'Spam' })
            >>> r3 = Record(attribute_map={u'id' : '1', u'title' : u'Eggs' })
            >>> r1 == r2
            ... False
            >>> r1.values_equal(r2)
            ... True
            >>> r1.values_equal(r3)
            ... False
        
        @param other: the Record to which this Record should be compared.
        @type other: Record
        @rtype: C{bool}
        '''
        try:
            return self.to_bibtex() == other.to_bibtex()
        except AttributeError:
            return False

    #{ methods for handling deprecated attributes

    def __getattr__(self, name):
        '''Handles attribute access for deprecated attributes and methods.
        
        @param name: the name of the requested attribute.
        @type name: str
        @rtype: C{object}
        @raise AttributeError:
            if this method does not handle the given attribute name.
        @note:
            'attribute' in this context refers to a python attribute,
            not to a BibJSON attribute.
        '''        
        if name=='citekey':
            warnings.warn(
                "The citekey attribute has been deprecated. " +
                "Please use the 'id' field instead.",
                DeprecationWarning, stacklevel=2)
            return self[u'id']
        if name=='bibtype':
            warnings.warn(
                "The bibtype attribute has been deprecated. " +
                "Please use the 'type' field instead.",
                DeprecationWarning, stacklevel=2)
            return self[u'type']
        
        deprecated_attributes = {
            'fields' : 'attribute_map',
            'field_names' : 'attribute_names'
        }
        if name in deprecated_attributes.keys():
            warnings.warn(
                'The ' + name + ' attribute has been deprecated. ' +
                'Please use "' + deprecated_attributes[name] + '" instead.',
                DeprecationWarning, stacklevel=2)
            try:
                return self.__dict__[deprecated_attributes[name]]
            except KeyError:
                # if deprecated_attributes[name] was not found in the
                # __dict__ for this instance then we assume it was a
                # method--and methods are stored as attributes of the class.
                return Record.__dict__[deprecated_attributes[name]].__get__(self, Record)
        raise AttributeError(name)
    
    def validate(self):
        '''Validates this record.
        @todo: add fuctionality.
        '''
        if self._parent_bib is not None:
            self._parent_bib.validate_record(self)
        
    @staticmethod
    def validate_attribute_name(attribute_name):
        '''Raises a BadAttributeNameError if the supplied string is not a valid attribute name.
        
        @param attribute_name: a potential attribute name
        @type attribute_name: C{Unicode}
        @rtype: C{None}
        @raises BadAttributeNameError:
            if attribute_name is not a valid attribute name.
        '''
        if not (isinstance(attribute_name, types.UnicodeType)):
            raise BadAttributeNameError('Attribute names must be Unicode strings')
        if attribute_name == '':
            raise BadAttributeNameError('Attribute names must not be empty')


class BibEntry(Record):
    '''(DEPRECATED) BibEntry is a alias for the L{Record} class.
    
    @deprecated:
        This class has been renamed 'Record' to match the vocabulary used
        by BibJSON.
    '''
    def __init__(self, citekey=None, bibtype='UNKNOWN', fields=None):
        '''Calls the __init__() method for the Record class and issues a DeprecationWarning.'''
        Record.__init__(self, citekey, bibtype, fields)
        warnings.warn(
            "The BibEntry class has been deprecated. " +
            "Please use the Record class instead.",
            DeprecationWarning, stacklevel=2)
        
        
class Bibliography:
    '''A bibliography class with methods to load BibTeX data.'''
    
    #{ Methods for initializing Bibliography instances
    
    def __init__(self,
                 id=None,
                 file=None,
                 string=None,
                 format=u'bibtex',
                 dlist=None,
                 preamble=None,
                 metadata=None,
                 entries=None):
        '''Initializes a new Bibliography instance.
        
        To create an empty Bibliography:
       
            >>> bib = Bibliography()
       
        To create a Bibliography from a file:
            
            >>> bib = Bibliography(file='articles.bib')

        or

            >>> file_handle = codecs.open('articles.bib', 'r', 'utf-8')
            >>> bib = Bibliography(file=file_handle)
        
        The file may be located on the web:
        
            >>> bib = Bibliography(file='example.org/books.bib')
            
        To create a Bibliography from a string:
            
            >>> bibstring = """
            ... @article{spam,
            ...   title = {The Spanish Inquisition},
            ...   note = {Nobody expects it!}
            ... }"""
            >>> bib = Bibliography(string=bibstring)
        
        To create a Bibliography from a list of dictionaries:
            
            >>> d_list = [{u'bibtype':u'article', 
            ...            u'citekey':u'spam',
            ...            u'title':u'The Spanish Inquisition'}]
            >>> bib = Bibliography(dlist = d_list)

        @param id:
            an identifier for this bibliography. If this is not supplied
            an identifier will be generated in an arbitrary manor.
        @type id: str
        @param file:
            a File object or the location of a file containing a
            serialization from which this Bibliography should load records.
        @type file: str or File
        @param string:
            a serialization from which this Bibliography should load records.
        @type string: str
        @param format:
            the serialization format used by the supplied string or
            file. Allowed values are 'bibtex' or 'bibjson'.
        @type format: str
        @param dlist:
            a list of dictionaries from which this Bibliography should
            load records.
        @type dlist: list
        @param preamble:
            LaTeX code to be included in the preamble of this
            bibliography if it is serialized in BibTeX format. (The
            preamble will appear as an attribute of the dataset module
            when serializing to BibJSON.)
        @type preamble: str
        @param metadata:
             metadata for this Bibliography. This will be included in the
             dataset module when serializing BibJSON.
        @type metadata: dict
        @param entries: a list of Records to be added to this Bibliography.
        @type entries: list
        '''
        ensure_objects_are_unicode(
            id, string, dlist, preamble, metadata, entries)
        self.id_to_record_index = {}
        '''A dict mapping local ids to entries. Used for quick-lookup.
        
        @type: C{dict}'''
        
        self.record_list = []
        '''A list of the Records in this Bibliography instance.
        
        @type: C{list}
        '''
        
        if metadata is None:
            self.metadata = {}
        else:
            self.metadata = metadata
        
        if id is None:
            self.metadata[u'id'] = unicode(repr(self), DEFAULT_ENCODING)
        else:
            self.metadata[u'id'] = id
        
        if preamble is None:
            self.metadata[u'bibtex_preamble'] = u''
        else:
           self.metadata[u'bibtex_preamble'] = preamble
        
        if entries is not None:
            for record in entries:
                self.add_record(record)
        
        if file is not None:
            self.load_from_file(file, format=format)
        
        if string is not None:
            self.load_from_string(string, format=format)
        
        if dlist is not None:
            self.load_from_dlist(dlist)
        
        if self.metadata['id'] in bib_cache.keys():
            warnings.warn(
                BKNBibtexWarning(
                    'A bibliography already exists with id ' + self.metadata['id']),
                stacklevel=2
                )
                # TO DO: subclass BKNBibtexWarning
        bib_cache[self.metadata['id']] = self

    def load_from_file(self, file, format='bibtex'):
        '''Loads bibliography entries from a file in the specified
        format.

        Loaded BibEntries are appended to any BibEntries already in the
        Bibliography.

        @param file:
            Specifies the file to use. May be a file object or a string.
            If a string is used, and it begins with 'http://' the file
            will be read from the web, otherwise a local file will be
            used.
        @type file:
            C{str} or C{File}
        @param format:
            The format in which the bibliographic data has been serialized.
        @type format:
            C{str}
        @rtype: C{None}
        @raises FormatNotSupportedError:
            A FormatNotSupportedError will be raised if the supplied format
            is not supported.
            
        @todo:
            consider revising this API. It may be better to do away with
            the format keyword and to use separate methods instead. 
            Another possibility would be to make this and similar methods
            class methods which return a Bibliography object.
        '''
        try:
            if file.startswith(u'http://') or file.startswith(u'file://'):
                self.metadata[u'id'] = file + u'#'
                file = urllib.urlopen(file)
            else:
                self.metadata[u'id'] = u'file://' + file + u'#'
        except AttributeError:
           self.metadata[u'id'] = u'file://' + file.name  + u'#'
        if format == u'bibtex': self._load_from_bibtex_file(file)
        elif format == u'bibjson': self._load_from_BibJSON_file(file)
        else: raise FormatNotSupportedError(format)
    
    def load_from_string(self, string, format='bibtex'):
        '''Loads bibliography entries from a string in the specified
        format.

        Loaded BibEntries are appended to any BibEntries already in the
        Bibliography.

        @param string:
            The string from which to load the records.
        @type string:
            C{str}
        @param format:
            The format in which the bibliographic data has been serialized.
        @type format:
            C{str}
        @rtype: C{None}
        @raises FormatNotSupportedError:
            A FormatNotSupportedError will be raised if the supplied format
            is not supported.
            
        @todo:
            consider revising this API. It may be better to do away with
            the format keyword and to use separate methods instead.
        '''
        ensure_objects_are_unicode(string)
        if format == 'bibtex': self._load_from_bibtex_string(string)
        elif format == 'bibjson': self._load_from_BibJSON_string(string)
        else: raise FormatNotSupportedError(format)
             
    def load_from_dlist(self, dlist):
        '''Loads bibliography entries from a list of dictionaries.
        
        This method will look for dictionaries of 'bibtype' and
        'citekey' to assign those parts of the entry. An error will be
        raised if either of these keys are missing.

        Loaded BibEntries are appended to any BibEntries already in the
        Bibliography.
        @param dlist:
            a list of dictionaries, with each dictionary corresponding
            to a record.
        @type dlist: C{list} of C{dict}s
        @rtype: C{None}
        '''
        for dict in dlist:
            d = copy.deepcopy(dict)
            try:
                type = d[u'type']
            except KeyError:
                type = d[u'bibtype'] # obsolete but still supported
                del d[u'bibtype']
            try:
                id = d[u'id']
            except KeyError:
                id = d[u'citekey'] # obsolete but still supported
                del d[u'citekey']
            e = Record(bibtype=type,
                citekey=id,
                attribute_map=d)
            self.add_record(e)

    def _load_from_bibtex_file(self, file, encoding='utf-8'):
        '''Loads records into this bibliography from the specified BibTeX file.
        
        @param file: the path to the file or a file object.
        @type file: C{str}, C{unicode}, or C{file}
        @rtype: C{None}
        @raise ParserSyntaxError: if there is an error in the BibTeX file.
        @todo:
            the encoding keyword is not currently handled correctly if
            C{file} is a C{File} object (as opposed to a file name). It
            may be best to split these two approaches into distinct
            methods.
        '''
        if isinstance(file, types.StringTypes):
            file = codecs.open(file, 'r', encoding)
            content = file.read()
        else:
            content = file.read().decode(encoding)
        try:
            self._load_from_bibtex_string(content)
        except ParserSyntaxError as err:
            raise BibFileSyntaxError(err, file.name)
    
    def _load_from_BibJSON_file(self, file, encoding='utf-8'):
        '''Loads records into this bibliography from the specified BibJSON file.
        
        @param file: the path to the file or a file object.
        @type file: C{str}, C{unicode}, or C{file}
        @rtype: C{None}
        @warning:
            This method uses the simpleparse module which does not support
            Unicode strings. For this reason strings passed to this method
            will be converted to the us-ascii encoding before being parsed.
        @todo:
            the encoding keyword is not currently handled correctly if
            C{file} is a C{File} object (as opposed to a file name). It
            may be best to split these two approaches into distinct
            methods.
        '''
        if isinstance(file, types.StringTypes):
            file = codecs.open(file, 'r', encoding)
            content = file.read()
        else:
            content = file.read().decode(encoding)
        self._load_from_BibJSON_string(content)
        
    def _load_from_BibJSON_string(self, string):
        '''Loads records into this bibliography from a string in BibJSON format.
        
        This method also updates the bibliography's metadata using the
        attributes of the supplied BibJSON's dataset module.
        
        @param string: BibJSON data
        @type string: C{str}
        @rtype: C{None}
        @todo: add support for linkage?
        @todo: add support for schema?
        '''
        ensure_objects_are_unicode(string)
        bibJSONroot = json.loads(string)
        self.metadata.update(bibJSONroot['dataset'])
        for e in bibJSONroot['recordList']:
            self.add_record(Record(attribute_map=e))

    def _load_from_bibtex_string(self, string):
        '''Loads records into this bibliography from a string in BibTeX format.
        
        @param string: BibTeX data
        @type string: C{str}
        @rtype: C{None}
        @warning:
            This method uses the simpleparse module which does not support
            Unicode strings. For this reason strings passed to this method
            will be converted to the us-ascii encoding before being parsed.
        '''
        ensure_objects_are_unicode(string)
        ascii_codec = codecs.lookup('us-ascii')
        (string, _) = ascii_codec.encode(string, 'replace') # second part of tuple is not needed
        result = BIBTEX_PARSER.parse(string, processor=BibtexProcessor())
        self.metadata[u'bibtex_preamble'] = unicode(result['preamble'], DEFAULT_ENCODING)
        for e in result['entries']:
            (bibtype, citekey, fields) = e
            attribute_map = {}
            for name, value in fields.items():
                u_name = unicode(name, DEFAULT_ENCODING)
                u_value = unicode(value, DEFAULT_ENCODING)
                attribute_map[u_name] = u_value
            self.add_record(
              Record(
                bibtype=unicode(bibtype, DEFAULT_ENCODING),
                citekey=unicode(citekey, DEFAULT_ENCODING),
                attribute_map=attribute_map
                )
              )
    
    #{ Emulating contatiner type
    def __len__(self):
        '''Returns the number of records in this bibliography.
        
        @rtype: C{int}
        @return: the number of records in this bibliography.
        '''
        return len(self.record_list)
    
    def __iter__(self):
        '''Returns an iterator over this bibliography's records.
        
        @rtype: C{iterator}
        @return: an iterator over this bibliography's records.
        '''
        return self.record_list.__iter__()

    def sort(self, comparison):
        '''Sort this bibliography, using the supplied comparison.
        
        Some useful comparison functions can be found in the
        L{comparisons} module.
        
             >>> from bkn.bibtex.comparisons import compare_entries
             >>> from bkn.bibtex.bibliography import Bibliography
             >>> bib = Bibliography(file='dataset.bib')
             >>> bib.sort(compare_entries)
        
        @param comparison:
            a comparison function taking two arguments which should
            return a negative or positive number depending on whether
            the first argument should come before or after the second
            argument.
        @type comparison: C{function}
        @rtype: C{None}
        '''
        self.record_list.sort(comparison)
    
    def __getitem__(self, key):
        '''Allows you to access a record using bracket notation.
        
            >>> bib['id:23']
       
        C{key} may be either an integer, in which case it refers to the
        position of a record in the bibliography, or it may be a string,
        in which case it refers to the local id of the record to be
        retrieved.
       
        Note that while you may use the bracket notation to access records
        you may not set records this way, so:
           
            >>> bib[0] = Record(record_data)
       
        will not work.
       
        @param key:
            either the position of the record to be retrieved or the
            local id of the record to be retrieved.
        @type key: C{int} or C{str}
        @return: the requested record
        @rtype: L{Record}
        '''
        ensure_objects_are_unicode(key)
        try:
            return self.record_list[key]
        except TypeError:
            return self.id_to_record_index[key]
    
    def append(self, record):
        '''A synomnym for add_record().
        
        @rtype: L{Record}
        @see: L{add_record()}
        '''
        self.add_record(record)
    
    def remove(self, record):
        '''A synonym for remove_record().
        
        @rtype: C{None}
        @raises RecordNotFoundError:
            if the specified record cannot be found in this Bibliography.
        @see: L{remove_record}
        '''
        self.remove_record(record)
    
    def add_record(self, record, id=None):
        '''Adds a copy of the supplied Record to this bibliography.
        
        If the keyword paramater C{id} is supplied it will be used as
        the local id for the new record. The original record is left
        unchanged.
        
        @param record: the Record to add
        @type record: L{Record}
        @return: the new Record
        @rtype: L{Record}
        @note:
            It is important to remember that a the supplied Record will
            not be added to the Bibliography; a copy of the Record will
            be added instead. This is necessary because a given Record
            can be a member of one and only one Bibliography.
        '''
        record_copy = copy.deepcopy(record)
        record_copy.set_parent_bib(self)

        if not id is None:
            record_copy[u'id'] = id
            
        # check for duplicate id
        if record_copy[u'id'] in self.id_to_record_index.keys():
            if record_copy is not self.id_to_record_index[record_copy[u'id']]:
                raise BKNBibtexError(
                    'A record with id ' + repr(record_copy[u'id']) +
                    ' already exists!')
                # should subclass BKNBibtexError
        
        # validate record
        self.validate_record(record_copy)
        
        # add to index
        self.id_to_record_index[record_copy[u'id']] = record_copy
        
        # add to this bib
        self.record_list.append(record_copy)
        return record_copy
    
    def remove_record(self, record):
        '''Removes the supplied Record from the bibliography.
        
        @param record: the record to be removed
        @type record: Record
        @rtype: C{None}
        @raises RecordNotFoundError:
            if the specified record cannot be found in this Bibliography.
        '''
        if record.get_parent_bib() != self:
            raise RecordNotFoundError('Record to be removed is not in this bibliography.')
        
        del self.id_to_record_index[record[u'id']]
        self.record_list.remove(record)
    
    #{ Methods for working with record ids and references
    
    def assign_numerical_ids(self):
        '''Assigns sequention numerical local ids to each record in this bibliography.'''
        # we must first assign non-numerics ids in order to avoid conflicts
        for i in range(len(self.record_list)):
            self.record_list[i][u'id'] = self.get_unique_id(prefix='tmp')
        for i in range(len(self.record_list)):
            self.record_list[i][u'id'] = unicode(i)

    def record_from_id(self, local_id):
        '''Returns the Record with the given local id.
        
        @param local_id: the local id of a record in this bibliography.
        @type local_id: str
        @return: the Record with the given local id
        @rtype: L{Record}
        @raises RecordNotFoundError:
            if no Record in this Bibliography has the specified local id.
        @warning:
            Do not confuse this method with the module level method
            L{record_from_id()<bkn.bibtex.bibliography.record_from_id>}.
        '''
        if local_id in self.id_to_record_index.keys():
            return self.id_to_record_index[local_id]
        else:
            raise RecordNotFoundError(
                'No record with the id ' + local_id +
                ' in this bibliography')

    def resolve_reference(self, ref, resolve_local_refs_from_memory=True):
        '''Attempts to resolve the given reference, returning the applicable Record if available.
        
        Global references will always be resolved over the network or
        from local files, as appropriate. Local references will be
        resolved from the version of the Bibliography in memory if
        resolve_local_refs_from_memory is True (the default). Otherwise
        they will be converted to and resolved like any other global
        reference.
        
        @param ref:
            The reference to be resolved. Global references should begin
            with '@@' and local references should begin with '@'.
        @type ref: str
        @param resolve_local_refs_from_memory:
            whether or not to resolve local references using the version
            of the Bibliography in memory.
        @type resolve_local_refs_from_memory: bool
        @return:
             the requested Record or None if the reference is cannot be
             resolved.
        @rtype: L{Record} or C{None}
        @raise InvalidReferenceError:
             if the requested reference was invalid.
        @raise RecordNotFoundError:
             if the requested reference can be resolved, and a response
             is retrieved, but the response does not contain the
             requested resource.
        @raise InvalidResponseError:
             if the requested reference can be resolved, and a response
             is retrieved, but the response is invalid.
        '''
        if ref.startswith('@@'):
            # global reference
            # global references must be URI's
            full_id = ref[2:] # <-- trim '@@'
            return record_from_id(full_id)
        elif ref.startswith('@'):
            # local reference
            local_id = ref[1:] # remove '@'
            if resolve_local_refs_from_memory:
                return self.record_from_id(local_id)
            else:
                full_id = self.metadata[u'id'] + local_id
                return self.resolve_reference('@@' + full_id)
        else:
            raise BKNBibtexError(
                '"' + ref + '" is not a valid reference. ' +
                'Valid references should begin with an "@".')
    
    def fullref(self, ref):
        '''Takes a reference, local or global, and returns a global reference.
        
        @note:
            a reference is simply a string composed of '@' plus the
            local id of a record or '@@' plus the global id of a record.
        @param ref: a reference
        @type ref: C{str}
        @return: a global reference
        @rtype: C{str}
        '''
        if ref.startswith(u'@@'):
            return ref
        else:
           return u'@@' + self.metadata[u'id'] + ref[1:]
    
    def get_records_same_as(self, id):
        '''Identifies records which represent the same thing by following same-as assertions.
        
        Same-as assertions may be included in a record by providing an
        attribute with the name "same-as" and a list of references as
        the value. For example, the same-as assertion in the this
        BibJSON record::
        
            {
                "id" : "003",
                "type" : "article",
                "title" : "Duplicate Detection in Underwater Steelworks",
                "author" : "Mandavis Grazy",
                "same-as" : ["@001","@002","@@http://external.data/978"]
            }
            
        indicates that the given record represents the same article as
        is represented by record 001 and 002 in this dataset as well as
        the record with full id http://external.data/978 which may be in
        an external dataset.
        
        Same-as assertions are symmetric. This method attempts to respect
        that, but same-as assertions must be indicated somewhere within
        a dataset in order to be observed by this method, i.e.
        assertions that only occur in external datasets will not be
        followed.
        
        @param id: the id of record to be considered.
        @type id: C{str}
        @return: a list of L{Records<Record>}
        @rtype: C{list} of L{Records<Record>}
        @attention:
            this method takes the record in question's id as an
            argument, not the record itself.
        '''
        result = []
        for record in self:
            if record.global_id() == id:
                result.append(record) # include the record itself
                if not record.attribute_is_blank(u'same-as'):
                    for r in record[u'same-as']:
                        result.append(self.resolve_reference(r))
                continue
            same_as_refs = [self.fullref(r) for r in record[u'same-as']]
            if u'@@' + id in same_as_refs:
               result.append(record)
        return result

    def get_unique_id(self, prefix=''):
        '''Returns a string for use as an id in this bibliography.
        
        @param prefix:
            if provided, the id will begin with the given prefix
        @type prefix: C{str}
        @return:
            a string which may be used as an id for a new record without
            conflicting with existing records in this Bibliography.
        @rtype: C{str}
        @todo: update tests for this method (the prefix keyword is new)
        '''
        try:
            self._unique_id_counter += 1
        except AttributeError:
            self._unique_id_counter = 0
        candidate_id = prefix + unicode(self._unique_id_counter)
        while candidate_id in self.id_to_record_index.keys():
            self._unique_id_counter += 1
            candidate_id = prefix + unicode(self._unique_id_counter)
        return candidate_id
    
    #{ Serialization methods
    
    def to_bibjson(self):
        '''Returns the BibJSON serialization of this Bibliography.
        
        @returns: the BibJSON serialization of this Bibliography.
        @rtype: C{str}
        '''
        dataset = {}
        dataset.update(self.metadata)
        record_list = []
        for record in self:
            record_list.append(record.attribute_map)
        result = { 'dataset' : dataset, 'recordList' : record_list }
        return json.dumps(result, sort_keys=True, indent=4)
        
    def to_bibtex(self):
        '''Returns the BibTeX serialization of this Bibliography.
        
        @returns: the BibTeX serialization of this Bibliography.
        @rtype: C{str}
        @warning:
             Bibliographies with any attribute values which are not strings
             or integers may not be serialized in the BibTeX format.
        @todo: write tests for this method.
        '''
        preamble = ''
        if not self.metadata.has_key(u'bibtex_preamble'):
            preamble = '@preamble{ "' + self.metadata[u'bibtex_preamble'] + '" }\n'
        formatted_entries = [e.to_bibtex() for e in self]
        return preamble + '\n'.join(formatted_entries)    
    
    #{ Enhancing and modifying Bibliographies
    
    def set_id(self, id):
        '''The preferred way to set this bibliographies id. Other methods
        may interfere with proper bibliography caching.
        
        @param id: the new id for this Bibliography.
        @type id: str
        @rtype: C{None}
        '''
        if not isinstance(id, types.StringTypes):
            raise BKNBibtexError('Dataset id must be a string')
        del bib_cache[self.metadata[u'id']]
        self.metadata[u'id'] = id
        bib_cache[self.metadata[u'id']] = self
            
    def add_bibtex_output(self, style='abbrv'):
        '''For every record, adds a bibtex_out attribute whose contents is
        the the result of running bibtex on the entry with specified
        style.

        A C{bibtex_warnings} attribute is also added, which is a list of any
        warnings generated by bibtex for an entry.
        
        @param style:
            the BibTeX style to be used. The specified style must be
            installed on your system.
        @type style: str
        @rtype: C{None}
        @raise ExternalError:
            if python can't find or run the bibtex executable.
        @raise ExternalError:
            if bibtex does not return the expected output.
        @raise BibTeXError:
            if bibtex encounters an error.
        @todo:
            this method should only attempt to create bibtex output for
            those records which represent bibliographic items. (Records
            for authors, institutions, and the like should be ignored.)
        '''
        _AddBibtexOutputEnchancer(self, style).compute()
                
    #{ Other methods
            
    def validate_record(self, record, raise_error=False):
        '''Returns True if the passed record is valid, otherwise returns False.
        
        @param raise_error:
            if C{True} then raise a InvalidRecordError instead of
            returning.
        @type raise_error: C{bool}
        @raise InvalidRecordError:
            if the record is invalid and C{raise_error} is C{True}
        @return:
            C{True} if the passed record is valid, and C{False}
            otherwise.
        @rtype: C{bool}
        @note:
            currently this method only checks that the 'type' and 'id'
            attributes are strings.
        @todo:
            should check against this bibliography's schema
        '''
        def fail(string):
            if raise_error:
                raise InvalidRecordError(string)
            else:
                return False
        # type must be a string
        for attr_name in [u'id', u'type']:
            if not isinstance(record[attr_name], types.StringTypes):
                fail('The ' + attr_name + ' attribute must take a string value.')
        return True
    
    def values_equal(self, other):
        '''Returns true if this Bibliography has the same content as the given Bibliography.
        
        Distinct Bibliographies will always compare as not equal. This method may
        be used to compare Records based upon their content.
        
            >>> bib2 = copy.deepcopy(bib1)
            >>> bib1 == bib2
            ... False
            >>> bib1.values_equal(bib2)
            ... True
        
        @param other: the Bibliography to which this Bibliography should be compared.
        @type other: Bibliography
        @rtype: C{bool}
        '''
        try:
            for i in range(len(self)):
                if not self[i].values_equal(other[i]):
                    return False
        except IndexError:
            return False
        except AttributeError:
            return False
        return True

    def __str__(self):
        '''Returns an informal string representation of this record
        
        @return:
            The current implementation calls to_bibtex(). (This may
            change in future versions.
        @rtype: C{str}
        '''
        return self.to_bibtex()

    def __getattr__(self, name):
        '''Handles attribute access for deprecated attributes and methods.
        
        @param name: the name of the requested attribute.
        @type name: str
        @rtype: C{object}
        @note:
            'attribute' in this context refers to a python attribute,
            not to a BibJSON attribute.
        '''                
        deprecated_attributes = {
            'entries' : 'record_list',
            'add_entry' : 'add_record',
            'remove_entry' : 'remove_record',
            'entry_from_id' : 'record_from_id'
        }
        if name in deprecated_attributes.keys():
            warnings.warn(
                'The ' + name + ' attribute has been deprecated. ' +
                'Please use "' + deprecated_attributes[name] + '" instead.',
                DeprecationWarning, stacklevel=2)
            try:
                return self.__dict__[deprecated_attributes[name]]
            except KeyError:
                # if deprecated_attributes[name] was not found in the
                # __dict__ for this instance then we assume it was a
                # method--and methods are stored as attributes of the class.
                return Bibliography.__dict__[deprecated_attributes[name]].__get__(self, Bibliography)
        raise AttributeError(name)
        
class _AddBibtexOutputEnchancer():
    '''A method object for adding the output of the BibTeX executable to a Bibliography.
    
    This class is used by Bibliography's add_bibtex_output() method.
    '''
    def __init__(self, bib, style):
        '''Creates a new _AddBibtexOutputEnchancer using the supplied Bibliography and style.
        
        @param bib: the Bibliography to enhance.
        @type bib: L{Bibliography}
        @param style:
            the BibTeX style to be used. The specified style must be
            installed on your system.
        @type style: str
        '''
        self.bib = bib
        self.style = style
        self.tmp_bib = copy.deepcopy(bib)
    
    def compute(self):
        '''Causes _AddBibtexOutputEnchancer to do its magic.
        
            >>> _AddBibtexOutputEnchancer(some_bib, 'abbrv').compute()
        '''
        if not which('bibtex'):
            raise ExternalError('bibtex executable is not available.')
        self.tmp_bib.assign_numerical_ids()
        self._prepare_temp_files()
        try:
            self._create_aux_file()
            self._create_bib_file()
            (stdout, bbl) = self._run_bibtex()
            warnings_dict = self._extract_bibtex_warnings(stdout)
            bbl_dict = self._extract_bibitems(bbl)
            self._enchance_with_warnings(warnings_dict)
            self._enchance_with_citations(bbl_dict)                
        finally:
            self._clean_up_temp_files()
    
    def _enchance_with_warnings(self, warnings_dict):
        for i in range(len(self.bib)):
            key = i.__str__()
            if warnings_dict.has_key(key):
                self.bib.record_list[i].bibtex_warnings = warnings_dict[key]
            else:
                self.bib.record_list[i].bibtex_warnings = ''
    
    def _enchance_with_citations(self, bbl_dict):
        for i in range(len(self.bib)):
            key = i.__str__()
            self.bib.record_list[i].bibtex_out = bbl_dict[key]
    
    def _create_aux_file(self):
        with codecs.open(self.aux_file, 'w', 'ascii') as f:
            f.write('''\\citation{*}\n\\bibdata{tmp}\n\\bibstyle{''' + self.style + '}')
    
    def _create_bib_file(self):
        with codecs.open(self.bib_file, 'w', DEFAULT_ENCODING) as f:
            f.write(self.tmp_bib.to_bibtex())
    
    def _run_bibtex(self):
        sp = subprocess.Popen(
          ['bibtex', 'tmp'],
          stdout=subprocess.PIPE
          )
        sp.wait()
        if sp.returncode != 0:
            raise BibTeXError(
               'Running bibtex failed. BibTeXoutput follows: ' +
               sp.stdout.read()
               )
        if not os.path.exists(self.bbl_file):
            raise ExternalError('bibtex failed to produce bbl_file')
        
        with codecs.open(self.bbl_file, 'r', DEFAULT_ENCODING) as f:
            bbl = f.read()
            
        return (sp.stdout, bbl)
    
    def _extract_bibtex_warnings(self, bibtex_output):
        result = {}
        for line in bibtex_output:
            m = BIBTEX_WARNING_REGEX.match(line)
            if m:
                (warning, key) = m.group(1, 2)
                result.setdefault(key,[]).append(warning)
        return result
    
    def _extract_bibitems(self, bbl):
        result = {}
        bbl = bbl.split(r'\end{thebibliography}')[0]
        items = bbl.split(r'\bibitem{')[1:]
        for item in items:
            key, value = item.split('}',1)
            result[key] = value
        return result
    
    def _prepare_temp_files(self):
        self.tmp_dr = tempfile.mkdtemp()
        self.wrk_dr = os.getcwd() # we need to restore this when we finish!
        self.bbl_file = os.path.join(self.tmp_dr, 'tmp.bbl')
        self.bib_file = os.path.join(self.tmp_dr, 'tmp.bib')
        self.aux_file = os.path.join(self.tmp_dr, 'tmp.aux')
        self.blg_file = os.path.join(self.tmp_dr, 'tmp.blg')
        os.chdir(self.tmp_dr)
    
    def _clean_up_temp_files(self):
        # We must enclose these os.remove commands in try clauses to
        # prevent errors here from masking earlier problems
        try:
            os.remove(self.bbl_file)
        except OSError:
            pass
        try:
            os.remove(self.aux_file)
        except OSError:
            pass
        try:
            os.remove(self.bib_file)
        except OSError:
            pass
        try:
            os.remove(self.blg_file)
        except OSError:
            pass
        os.removedirs(self.tmp_dr)
        os.chdir(self.wrk_dr)  
